Set SM2 error codes

Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/6386)

3 files changed, 235 insertions, 32 deletions

diff --git a/crypto/sm2/sm2_crypt.c b/crypto/sm2/sm2_crypt.c
index c3abd969eb..fc7949d272 100644
--- a/crypto/sm2/sm2_crypt.c
+++ b/crypto/sm2/sm2_crypt.c
@@ -10,6 +10,8 @@
  */
 
 #include "internal/sm2.h"
+#include "internal/sm2err.h"
+#include <openssl/err.h>
 #include <openssl/evp.h>
 #include <openssl/bn.h>
 #include <openssl/asn1.h>
@@ -112,11 +114,17 @@ int SM2_encrypt(const EC_KEY *key,
     kG = EC_POINT_new(group);
     kP = EC_POINT_new(group);
     if (kG == NULL || kP == NULL)
+       {
+       SM2err(SM2_F_SM2_ENCRYPT, ERR_R_MALLOC_FAILURE);
        goto done;
+       }
 
     ctx = BN_CTX_new();
     if (ctx == NULL)
+       {
+       SM2err(SM2_F_SM2_ENCRYPT, ERR_R_MALLOC_FAILURE);
        goto done;
+       }
 
     BN_CTX_start(ctx);
     k = BN_CTX_get(ctx);
@@ -126,36 +134,57 @@ int SM2_encrypt(const EC_KEY *key,
     y2 = BN_CTX_get(ctx);
 
     if (y2 == NULL)
-       goto done;
+        {
+        SM2err(SM2_F_SM2_ENCRYPT, ERR_R_BN_LIB);
+        goto done;
+        }
 
     x2y2 = OPENSSL_zalloc(2 * field_size);
     C3 = OPENSSL_zalloc(C3_size);
 
     if (x2y2 == NULL || C3 == NULL)
-       goto done;
+        {
+        SM2err(SM2_F_SM2_ENCRYPT, ERR_R_MALLOC_FAILURE);
+        goto done;
+        }
 
     memset(ciphertext_buf, 0, *ciphertext_len);
 
     BN_priv_rand_range(k, order);
 
     if (EC_POINT_mul(group, kG, k, NULL, NULL, ctx) == 0)
+        {
+        SM2err(SM2_F_SM2_ENCRYPT, ERR_R_EC_LIB);
         goto done;
+        }
 
     if (EC_POINT_get_affine_coordinates_GFp(group, kG, x1, y1, ctx) == 0)
+        {
+        SM2err(SM2_F_SM2_ENCRYPT, ERR_R_EC_LIB);
         goto done;
+        }
 
     if (EC_POINT_mul(group, kP, NULL, P, k, ctx) == 0)
+        {
+        SM2err(SM2_F_SM2_ENCRYPT, ERR_R_EC_LIB);
         goto done;
+        }
 
     if (EC_POINT_get_affine_coordinates_GFp(group, kP, x2, y2, ctx) == 0)
+        {
+        SM2err(SM2_F_SM2_ENCRYPT, ERR_R_EC_LIB);
         goto done;
+        }
 
     BN_bn2binpad(x2, x2y2, field_size);
     BN_bn2binpad(y2, x2y2 + field_size, field_size);
 
     msg_mask = OPENSSL_zalloc(msg_len);
     if (msg_mask == NULL)
+       {
+       SM2err(SM2_F_SM2_ENCRYPT, ERR_R_MALLOC_FAILURE);
        goto done;
+       }
 
     /* X9.63 with no salt happens to match the KDF used in SM2 */
     if (ECDH_KDF_X9_62(msg_mask, msg_len, x2y2, 2 * field_size, NULL, 0, digest)
@@ -166,19 +195,34 @@ int SM2_encrypt(const EC_KEY *key,
         msg_mask[i] ^= msg[i];
 
     if (EVP_DigestInit(hash, digest) == 0)
+        {
+        SM2err(SM2_F_SM2_ENCRYPT, ERR_R_EVP_LIB);
         goto done;
+        }
 
     if (EVP_DigestUpdate(hash, x2y2, field_size) == 0)
+        {
+        SM2err(SM2_F_SM2_ENCRYPT, ERR_R_EVP_LIB);
         goto done;
+        }
 
     if (EVP_DigestUpdate(hash, msg, msg_len) == 0)
+        {
+        SM2err(SM2_F_SM2_ENCRYPT, ERR_R_EVP_LIB);
         goto done;
+        }
 
     if (EVP_DigestUpdate(hash, x2y2 + field_size, field_size) == 0)
+        {
+        SM2err(SM2_F_SM2_ENCRYPT, ERR_R_EVP_LIB);
         goto done;
+        }
 
     if (EVP_DigestFinal(hash, C3, NULL) == 0)
+        {
+        SM2err(SM2_F_SM2_ENCRYPT, ERR_R_EVP_LIB);
         goto done;
+        }
 
     ctext_struct.C1x = x1;
     ctext_struct.C1y = y1;
@@ -240,10 +284,16 @@ int SM2_decrypt(const EC_KEY *key,
     sm2_ctext = d2i_SM2_Ciphertext(NULL, &ciphertext, ciphertext_len);
 
     if (sm2_ctext == NULL)
+        {
+        SM2err(SM2_F_SM2_DECRYPT, SM2_R_ASN1_ERROR);
         goto done;
+        }
 
     if (sm2_ctext->C3->length != hash_size)
+        {
+        SM2err(SM2_F_SM2_DECRYPT, SM2_R_INVALID_ENCODING);
         goto done;
+        }
 
     C2 = sm2_ctext->C2->data;
     C3 = sm2_ctext->C3->data;
@@ -251,36 +301,57 @@ int SM2_decrypt(const EC_KEY *key,
 
     ctx = BN_CTX_new();
     if (ctx == NULL)
-       goto done;
+        {
+        SM2err(SM2_F_SM2_DECRYPT, ERR_R_MALLOC_FAILURE);
+        goto done;
+        }
 
     BN_CTX_start(ctx);
     x2 = BN_CTX_get(ctx);
     y2 = BN_CTX_get(ctx);
 
     if(y2 == NULL)
-       goto done;
+        {
+        SM2err(SM2_F_SM2_DECRYPT, ERR_R_BN_LIB);
+        goto done;
+        }
 
     msg_mask = OPENSSL_zalloc(msg_len);
     x2y2 = OPENSSL_zalloc(2 * field_size);
     computed_C3 = OPENSSL_zalloc(hash_size);
 
     if(msg_mask == NULL || x2y2 == NULL || computed_C3 == NULL)
-       goto done;
+        {
+        SM2err(SM2_F_SM2_DECRYPT, ERR_R_MALLOC_FAILURE);
+        goto done;
+        }
 
     C1 = EC_POINT_new(group);
     if (C1 == NULL)
+        {
+        SM2err(SM2_F_SM2_DECRYPT, ERR_R_MALLOC_FAILURE);
         goto done;
+        }
 
     if (EC_POINT_set_affine_coordinates_GFp
         (group, C1, sm2_ctext->C1x, sm2_ctext->C1y, ctx) == 0)
+        {
+        SM2err(SM2_F_SM2_DECRYPT, ERR_R_EC_LIB);
         goto done;
+        }
 
     if (EC_POINT_mul(group, C1, NULL, C1, EC_KEY_get0_private_key(key), ctx) ==
         0)
+        {
+        SM2err(SM2_F_SM2_DECRYPT, ERR_R_EC_LIB);
         goto done;
+        }
 
     if (EC_POINT_get_affine_coordinates_GFp(group, C1, x2, y2, ctx) == 0)
+        {
+        SM2err(SM2_F_SM2_DECRYPT, ERR_R_EC_LIB);
         goto done;
+        }
 
     BN_bn2binpad(x2, x2y2, field_size);
     BN_bn2binpad(y2, x2y2 + field_size, field_size);
@@ -295,22 +366,40 @@ int SM2_decrypt(const EC_KEY *key,
     hash = EVP_MD_CTX_new();
 
     if (hash == NULL)
-       goto done;
+        {
+        SM2err(SM2_F_SM2_DECRYPT, ERR_R_MALLOC_FAILURE);
+        goto done;
+        }
 
     if (EVP_DigestInit(hash, digest) == 0)
+        {
+        SM2err(SM2_F_SM2_DECRYPT, ERR_R_EVP_LIB);
         goto done;
+        }
 
     if (EVP_DigestUpdate(hash, x2y2, field_size) == 0)
+        {
+        SM2err(SM2_F_SM2_DECRYPT, ERR_R_EVP_LIB);
         goto done;
+        }
 
     if (EVP_DigestUpdate(hash, ptext_buf, msg_len) == 0)
+        {
+        SM2err(SM2_F_SM2_DECRYPT, ERR_R_EVP_LIB);
         goto done;
+        }
 
     if (EVP_DigestUpdate(hash, x2y2 + field_size, field_size) == 0)
+        {
+        SM2err(SM2_F_SM2_DECRYPT, ERR_R_EVP_LIB);
         goto done;
+        }
 
     if (EVP_DigestFinal(hash, computed_C3, NULL) == 0)
+        {
+        SM2err(SM2_F_SM2_DECRYPT, ERR_R_EVP_LIB);
         goto done;
+        }
 
     if (memcmp(computed_C3, C3, hash_size) != 0)
         goto done;
diff --git a/crypto/sm2/sm2_err.c b/crypto/sm2/sm2_err.c
index 6f244a5eb0..ef3f5e6b1d 100644
--- a/crypto/sm2/sm2_err.c
+++ b/crypto/sm2/sm2_err.c
@@ -19,6 +19,14 @@ static const ERR_STRING_DATA SM2_str_functs[] = {
     {ERR_PACK(ERR_LIB_SM2, SM2_F_PKEY_SM2_KEYGEN, 0), "pkey_sm2_keygen"},
     {ERR_PACK(ERR_LIB_SM2, SM2_F_PKEY_SM2_PARAMGEN, 0), "pkey_sm2_paramgen"},
     {ERR_PACK(ERR_LIB_SM2, SM2_F_PKEY_SM2_SIGN, 0), "pkey_sm2_sign"},
+    {ERR_PACK(ERR_LIB_SM2, SM2_F_SM2_COMPUTE_MSG_HASH, 0),
+     "SM2_compute_msg_hash"},
+    {ERR_PACK(ERR_LIB_SM2, SM2_F_SM2_DECRYPT, 0), "SM2_decrypt"},
+    {ERR_PACK(ERR_LIB_SM2, SM2_F_SM2_ENCRYPT, 0), "SM2_encrypt"},
+    {ERR_PACK(ERR_LIB_SM2, SM2_F_SM2_SIGN, 0), "SM2_sign"},
+    {ERR_PACK(ERR_LIB_SM2, SM2_F_SM2_SIG_GEN, 0), "SM2_sig_gen"},
+    {ERR_PACK(ERR_LIB_SM2, SM2_F_SM2_SIG_VERIFY, 0), "SM2_sig_verify"},
+    {ERR_PACK(ERR_LIB_SM2, SM2_F_SM2_VERIFY, 0), "SM2_verify"},
     {0, NULL}
 };
 
diff --git a/crypto/sm2/sm2_sign.c b/crypto/sm2/sm2_sign.c
index ddfd318ed9..61f01eae6e 100644
--- a/crypto/sm2/sm2_sign.c
+++ b/crypto/sm2/sm2_sign.c
@@ -10,14 +10,17 @@
  */
 
 #include "internal/sm2.h"
+#include "internal/sm2err.h"
+#include <openssl/err.h>
 #include <openssl/evp.h>
+#include <openssl/err.h>
 #include <openssl/bn.h>
 #include <string.h>
 
-static BIGNUM *compute_msg_hash(const EVP_MD *digest,
-                                const EC_KEY *key,
-                                const char *user_id,
-                                const uint8_t *msg, size_t msg_len)
+static BIGNUM *SM2_compute_msg_hash(const EVP_MD *digest,
+                                    const EC_KEY *key,
+                                    const char *user_id,
+                                    const uint8_t *msg, size_t msg_len)
 {
     EVP_MD_CTX *hash = EVP_MD_CTX_new();
     const int md_size = EVP_MD_size(digest);
@@ -25,26 +28,44 @@ static BIGNUM *compute_msg_hash(const EVP_MD *digest,
     BIGNUM *e = NULL;
 
     if (za == NULL)
+        {
+        SM2err(SM2_F_SM2_COMPUTE_MSG_HASH, ERR_R_MALLOC_FAILURE);
         goto done;
+        }
 
     if (hash == NULL)
+        {
+        SM2err(SM2_F_SM2_COMPUTE_MSG_HASH, ERR_R_MALLOC_FAILURE);
         goto done;
+        }
 
     if (SM2_compute_userid_digest(za, digest, user_id, key) == 0)
         goto done;
 
     if (EVP_DigestInit(hash, digest) == 0)
+        {
+        SM2err(SM2_F_SM2_COMPUTE_MSG_HASH, ERR_R_EVP_LIB);
         goto done;
+        }
 
     if (EVP_DigestUpdate(hash, za, md_size) == 0)
+        {
+        SM2err(SM2_F_SM2_COMPUTE_MSG_HASH, ERR_R_EVP_LIB);
         goto done;
+        }
 
     if (EVP_DigestUpdate(hash, msg, msg_len) == 0)
+        {
+        SM2err(SM2_F_SM2_COMPUTE_MSG_HASH, ERR_R_EVP_LIB);
         goto done;
+        }
 
     /* reuse za buffer to hold H(ZA || M) */
     if (EVP_DigestFinal(hash, za, NULL) == 0)
+        {
+        SM2err(SM2_F_SM2_COMPUTE_MSG_HASH, ERR_R_EVP_LIB);
         goto done;
+        }
 
     e = BN_bin2bn(za, md_size, NULL);
 
@@ -73,11 +94,17 @@ ECDSA_SIG *SM2_sig_gen(const EC_KEY *key, const BIGNUM *e)
 
     kG = EC_POINT_new(group);
     if (kG == NULL)
+        {
+        SM2err(SM2_F_SM2_SIG_GEN, ERR_R_MALLOC_FAILURE);
         goto done;
+        }
 
     ctx = BN_CTX_new();
     if (ctx == NULL)
+        {
+        SM2err(SM2_F_SM2_SIG_GEN, ERR_R_MALLOC_FAILURE);
         goto done;
+        }
 
     BN_CTX_start(ctx);
 
@@ -87,26 +114,41 @@ ECDSA_SIG *SM2_sig_gen(const EC_KEY *key, const BIGNUM *e)
     tmp = BN_CTX_get(ctx);
 
     if (tmp == NULL)
+        {
+        SM2err(SM2_F_SM2_SIG_GEN, ERR_R_BN_LIB);
         goto done;
+        }
 
     /* These values are returned and so should not be allocated out of the context */
     r = BN_new();
     s = BN_new();
 
     if (r == NULL || s == NULL)
+        {
+        SM2err(SM2_F_SM2_SIG_GEN, ERR_R_MALLOC_FAILURE);
         goto done;
+        }
 
     for (;;) {
         BN_priv_rand_range(k, order);
 
         if (EC_POINT_mul(group, kG, k, NULL, NULL, ctx) == 0)
+            {
+            SM2err(SM2_F_SM2_SIG_GEN, ERR_R_EC_LIB);
             goto done;
+            }
 
         if (EC_POINT_get_affine_coordinates_GFp(group, kG, x1, NULL, ctx) == 0)
+            {
+            SM2err(SM2_F_SM2_SIG_GEN, ERR_R_EC_LIB);
             goto done;
+            }
 
         if (BN_mod_add(r, e, x1, order, ctx) == 0)
+            {
+            SM2err(SM2_F_SM2_SIG_GEN, ERR_R_BN_LIB);
             goto done;
+            }
 
         /* try again if r == 0 or r+k == n */
         if (BN_is_zero(r))
@@ -117,18 +159,43 @@ ECDSA_SIG *SM2_sig_gen(const EC_KEY *key, const BIGNUM *e)
         if (BN_cmp(rk, order) == 0)
             continue;
 
-        BN_add(s, dA, BN_value_one());
-        BN_mod_inverse(s, s, order, ctx);
-
-        BN_mod_mul(tmp, dA, r, order, ctx);
-        BN_sub(tmp, k, tmp);
-
-        BN_mod_mul(s, s, tmp, order, ctx);
+        if(BN_add(s, dA, BN_value_one()) == 0)
+           {
+           SM2err(SM2_F_SM2_SIG_GEN, ERR_R_BN_LIB);
+           goto done;
+           }
+
+        if(BN_mod_inverse(s, s, order, ctx) == 0)
+           {
+           SM2err(SM2_F_SM2_SIG_GEN, ERR_R_BN_LIB);
+           goto done;
+           }
+
+        if(BN_mod_mul(tmp, dA, r, order, ctx) == 0)
+           {
+           SM2err(SM2_F_SM2_SIG_GEN, ERR_R_BN_LIB);
+           goto done;
+           }
+
+        if(BN_sub(tmp, k, tmp) == 0)
+           {
+           SM2err(SM2_F_SM2_SIG_GEN, ERR_R_BN_LIB);
+           goto done;
+           }
+
+        if(BN_mod_mul(s, s, tmp, order, ctx) == 0)
+           {
+           SM2err(SM2_F_SM2_SIG_GEN, ERR_R_BN_LIB);
+           goto done;
+           }
 
         sig = ECDSA_SIG_new();
 
         if (sig == NULL)
+            {
+            SM2err(SM2_F_SM2_SIG_GEN, ERR_R_MALLOC_FAILURE);
             goto done;
+            }
 
          /* takes ownership of r and s */
         ECDSA_SIG_set0(sig, r, s);
@@ -164,10 +231,17 @@ int SM2_sig_verify(const EC_KEY *key, const ECDSA_SIG *sig, const BIGNUM *e)
 
     ctx = BN_CTX_new();
     if (ctx == NULL)
+        {
+        SM2err(SM2_F_SM2_SIG_VERIFY, ERR_R_MALLOC_FAILURE);
         goto done;
+        }
+
     pt = EC_POINT_new(group);
     if (pt == NULL)
+        {
+        SM2err(SM2_F_SM2_SIG_VERIFY, ERR_R_MALLOC_FAILURE);
         goto done;
+        }
 
     BN_CTX_start(ctx);
 
@@ -175,7 +249,10 @@ int SM2_sig_verify(const EC_KEY *key, const ECDSA_SIG *sig, const BIGNUM *e)
     x1 = BN_CTX_get(ctx);
 
     if (x1 == NULL)
-        goto done;
+       {
+       SM2err(SM2_F_SM2_SIG_VERIFY, ERR_R_MALLOC_FAILURE);
+       goto done;
+       }
 
     /*
        B1: verify whether r' in [1,n-1], verification failed if not
@@ -189,30 +266,47 @@ int SM2_sig_verify(const EC_KEY *key, const ECDSA_SIG *sig, const BIGNUM *e)
 
     ECDSA_SIG_get0(sig, &r, &s);
 
-    if (BN_cmp(r, BN_value_one()) < 0)
-        goto done;
-    if (BN_cmp(s, BN_value_one()) < 0)
+    if ((BN_cmp(r, BN_value_one()) < 0) || (BN_cmp(s, BN_value_one()) < 0))
+        {
+        SM2err(SM2_F_SM2_SIG_VERIFY, SM2_R_BAD_SIGNATURE);
         goto done;
+        }
 
-    if (BN_cmp(order, r) <= 0)
-        goto done;
-    if (BN_cmp(order, s) <= 0)
+    if ((BN_cmp(order, r) <= 0) || (BN_cmp(order, s) <= 0))
+        {
+        SM2err(SM2_F_SM2_SIG_VERIFY, SM2_R_BAD_SIGNATURE);
         goto done;
+        }
 
     if (BN_mod_add(t, r, s, order, ctx) == 0)
+        {
+        SM2err(SM2_F_SM2_SIG_VERIFY, ERR_R_BN_LIB);
         goto done;
+        }
 
     if (BN_is_zero(t) == 1)
+        {
+        SM2err(SM2_F_SM2_SIG_VERIFY, SM2_R_BAD_SIGNATURE);
         goto done;
+        }
 
     if (EC_POINT_mul(group, pt, s, EC_KEY_get0_public_key(key), t, ctx) == 0)
+        {
+        SM2err(SM2_F_SM2_SIG_VERIFY, ERR_R_EC_LIB);
         goto done;
+        }
 
     if (EC_POINT_get_affine_coordinates_GFp(group, pt, x1, NULL, ctx) == 0)
+        {
+        SM2err(SM2_F_SM2_SIG_VERIFY, ERR_R_EC_LIB);
         goto done;
+        }
 
     if (BN_mod_add(t, e, x1, order, ctx) == 0)
+        {
+        SM2err(SM2_F_SM2_SIG_VERIFY, ERR_R_BN_LIB);
         goto done;
+        }
 
     if (BN_cmp(r, t) == 0)
         ret = 1;
@@ -230,7 +324,7 @@ ECDSA_SIG *SM2_do_sign(const EC_KEY *key,
     BIGNUM *e = NULL;
     ECDSA_SIG *sig = NULL;
 
-    e = compute_msg_hash(digest, key, user_id, msg, msg_len);
+    e = SM2_compute_msg_hash(digest, key, user_id, msg, msg_len);
     if (e == NULL)
         goto done;
 
@@ -249,7 +343,7 @@ int SM2_do_verify(const EC_KEY *key,
     BIGNUM *e = NULL;
     int ret = -1;
 
-    e = compute_msg_hash(digest, key, user_id, msg, msg_len);
+    e = SM2_compute_msg_hash(digest, key, user_id, msg, msg_len);
     if (e == NULL)
         goto done;
 
@@ -267,11 +361,11 @@ int SM2_sign(int type, const unsigned char *dgst, int dgstlen,
     ECDSA_SIG *s = NULL;
     int ret = -1;
 
-    if (type != NID_sm3)
-        goto done;
-
-    if (dgstlen != 32)          /* expected length of SM3 hash */
-        goto done;
+    if (type != NID_sm3 || dgstlen != 32)
+       {
+       SM2err(SM2_F_SM2_SIGN, SM2_R_INVALID_DIGEST_TYPE);
+       goto done;
+       }
 
     e = BN_bin2bn(dgst, dgstlen, NULL);
 
@@ -298,17 +392,29 @@ int SM2_verify(int type, const unsigned char *dgst, int dgstlen,
     int ret = -1;
 
     if (type != NID_sm3)
-        goto done;
+       {
+       SM2err(SM2_F_SM2_VERIFY, SM2_R_INVALID_DIGEST_TYPE);
+       goto done;
+       }
 
     s = ECDSA_SIG_new();
     if (s == NULL)
+        {
+        SM2err(SM2_F_SM2_VERIFY, ERR_R_MALLOC_FAILURE);
         goto done;
+        }
     if (d2i_ECDSA_SIG(&s, &p, sig_len) == NULL)
+        {
+        SM2err(SM2_F_SM2_VERIFY, SM2_R_INVALID_ENCODING);
         goto done;
+        }
     /* Ensure signature uses DER and doesn't have trailing garbage */
     derlen = i2d_ECDSA_SIG(s, &der);
     if (derlen != sig_len || memcmp(sig, der, derlen) != 0)
+        {
+        SM2err(SM2_F_SM2_VERIFY, SM2_R_INVALID_ENCODING);
         goto done;
+        }
 
     e = BN_bin2bn(dgst, dgstlen, NULL);